var fs = require("fs")
var os = require("os");
const process = require('process');
// process.exit(1);
const csvParser = require('csv-parser');
const sourceFile = './csv/origin.csv';
const tagetFile = './csv/clean.csv';
const {keys} =  require('../config/key.js');

console.log('----- clean start -----');
fs.appendFileSync(tagetFile, keys.join(',') + os.EOL);
let ele = {};
fs.createReadStream(sourceFile)
  .pipe(csvParser())
  .on('data', (data) => {
    if (ele.time && ele.time != data.time) {
        writeFile(tagetFile, ele);
    } 
    ele = Object.assign({}, data);
  })
  .on('end', () => {
    writeFile(tagetFile, ele);
    console.log('----- clean ende -----');
  });


function writeFile(filePath, data){
    let arr = [];
    keys.forEach(val=>{
      arr.push(data[val]);
    })
    const text = arr.join(',')
	  fs.appendFileSync(filePath, text + os.EOL);
}
